#!/bin/bash
STREAM=/home/hadoop/hadoop/hadoop_install/contrib/streaming
#==============================================
MAPPER=/home/hadoop/progs/python/projects/rt/step3a_rt_combine/mapper.py
REDUCER=/home/hadoop/progs/python/projects/rt/step3a_rt_combine/reducer.py
FILE=/home/hadoop/progs/python/projects/rt/mylib.py
#==============================================
INPUT1=/home/lih307/projects/rt/step_2_tweets_0_md5.rt
INPUT2=/home/lih307/projects/rt/step_2_tweets_1_md5
OUTPUT=/home/lih307/projects/rt/step_3a_rt_combine
#==============================================
REDUCE_NUM=60
#=============================================
hadoop fs -rmr $OUTPUT
hadoop jar $STREAM/hadoop-streaming-1.0.3.jar \
    -D mapred.reduce.tasks=$REDUCE_NUM \
    -input $INPUT1 \
    -input $INPUT2 \
    -output $OUTPUT \
    -mapper $MAPPER \
    -reducer $REDUCER \
    -file $MAPPER \
    -file $REDUCER \
    -file $FILE